// 从mcp.so获取服务器数据的脚本
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import axios from 'axios';
import { JSDOM } from 'jsdom';

// 获取__dirname
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

// 创建目录（如果不存在）
const dataDir = path.join(__dirname, '../src/data');
if (!fs.existsSync(dataDir)) {
  fs.mkdirSync(dataDir, { recursive: true });
}

// 用于存储服务器数据的文件路径
const outputFilePath = path.join(dataDir, 'servers.json');

// 从MCP.so抓取数据
async function fetchServersFromMCP() {
  try {
    console.log('开始从mcp.so获取服务器数据...');
    
    // 使用更高级的爬取策略
    // 1. 首先获取主页内容
    const mainPageResponse = await axios.get('https://mcp.so/', {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
      }
    });
    
    const mainPageHtml = mainPageResponse.data;
    console.log('已获取主页内容，开始解析服务器列表...');
    
    // 使用JSDOM解析HTML
    const dom = new JSDOM(mainPageHtml);
    const document = dom.window.document;
    
    // 提取所有服务器链接
    // 我们查找页面中所有看起来像服务器链接的元素
    const serverLinks = Array.from(document.querySelectorAll('a[href]'))
      .filter(a => {
        const href = a.getAttribute('href');
        // 从主页上寻找指向特定服务器页面的链接
        return href && (
          href.includes('/server/') || 
          href.includes('/api/') || 
          href.includes('/mcp/') ||
          href.match(/\/[a-z0-9-]+$/)
        );
      })
      .map(a => {
        const href = a.getAttribute('href');
        // 处理相对链接
        const url = href.startsWith('http') ? href : `https://mcp.so${href.startsWith('/') ? href : `/${href}`}`;
        const name = a.textContent.trim() || path.basename(href);
        return { url, name };
      });
    
    console.log(`找到 ${serverLinks.length} 个可能的服务器链接`);
    
    // 如果找不到服务器链接，则使用备用方法解析页面内容
    if (serverLinks.length === 0) {
      console.log('无法找到服务器链接，尝试直接从页面内容提取...');
      
      // 提取所有可能包含服务器信息的部分
      const serverSections = Array.from(document.querySelectorAll('div.card, div.server, section, article'));
      const servers = [];
      
      serverSections.forEach((section, index) => {
        const title = section.querySelector('h1, h2, h3, h4, strong, .title');
        const description = section.querySelector('p, .description, .content');
        
        if (title) {
          const name = title.textContent.trim();
          const desc = description ? description.textContent.trim() : '无描述';
          
          // 尝试提取标签
          const tagElements = section.querySelectorAll('.tag, .badge, .label');
          const tags = Array.from(tagElements).map(tag => tag.textContent.trim());
          
          // 确定类型（官方/社区）
          const type = section.classList.contains('official') || 
                      section.classList.contains('primary') || 
                      title.classList.contains('official') ? 
                      'official' : 'community';
          
          servers.push({
            id: name.toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, ''),
            name,
            description: desc,
            provider: 'modelcontextprotocol',
            type,
            tags: tags.length > 0 ? tags : ['general'],
            documentation: `https://mcp.so/${name.toLowerCase().replace(/\s+/g, '-')}`
          });
        }
      });
      
      if (servers.length > 0) {
        console.log(`从页面内容提取了 ${servers.length} 个服务器信息`);
        return { servers };
      }
    }
    
    // 如果仍然找不到服务器，则需要尝试爬取GitHub仓库或其他相关资源
    console.log('尝试从GitHub仓库获取MCP服务器信息...');
    try {
      const githubResponse = await axios.get('https://api.github.com/search/repositories?q=mcp+server+protocol&sort=stars&order=desc', {
        headers: {
          'Accept': 'application/vnd.github.v3+json',
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36'
        }
      });
      
      const repos = githubResponse.data.items;
      console.log(`从GitHub找到 ${repos.length} 个相关仓库`);
      
      if (repos.length > 0) {
        const servers = repos.map(repo => {
          // 从仓库名称和描述提取信息
          const name = repo.name.replace(/-/g, ' ').replace(/mcp/i, '').replace(/server/i, '').trim() || repo.name;
          
          return {
            id: repo.name.toLowerCase(),
            name: name.charAt(0).toUpperCase() + name.slice(1), // 首字母大写
            description: repo.description || `GitHub仓库: ${repo.full_name}`,
            provider: repo.owner.login,
            type: repo.owner.type === 'Organization' ? 'official' : 'community',
            tags: ['github', repo.language ? repo.language.toLowerCase() : 'general'],
            url: repo.html_url,
            github: repo.html_url,
            documentation: repo.homepage || repo.html_url,
            language: repo.language ? [repo.language] : []
          };
        });
        
        return { servers };
      }
    } catch (error) {
      console.error('从GitHub获取信息时出错:', error.message);
    }
    
    // 如果所有方法都失败，则生成更丰富的示例数据
    console.log('无法从在线源获取服务器数据，使用示例数据');
    return generateSampleData(150); // 生成更多示例数据
  } catch (error) {
    console.error('获取服务器数据时发生错误:', error);
    console.log('使用示例数据作为备份。');
    return generateSampleData(150);
  }
}

// 生成示例数据
function generateSampleData(count = 30) {
  // 扩展现有的示例数据
  const tags = [
    'storage', 'api', 'search', 'database', 'scraping', 'automation', 'security', 
    'analytics', 'ai', 'nlp', 'vector', 'image', 'audio', 'video', 'generative',
    'translation', 'knowledge', 'code', 'chat', 'memory', 'reasoning', 'social',
    'tools', 'utility', 'cloud', 'data', 'finance', 'healthcare', 'research',
    'education', 'gaming', 'media', 'content', 'streaming'
  ];
  
  const types = ['official', 'community'];
  const languages = ['JavaScript', 'Python', 'TypeScript', 'Go', 'Rust', 'Java', 'C#', 'Ruby', 'PHP', 'Swift'];
  
  // 预定义服务器名称列表（扩展更多真实感的名称）
  const serverNames = [
    // 存储和数据库相关
    "Filesystem", "PostgreSQL", "MongoDB", "Redis Cache", "Vector DB", "Document Store",
    "GraphQL API", "SQL Query", "Data Lake", "Time Series DB", "Object Storage",
    
    // API和集成相关
    "GitHub", "GitLab", "Bitbucket", "Brave Search", "Google API", "OpenAI Completion",
    "Azure Cognitive", "AWS Integration", "Anthropic API", "Hugging Face", "Gemini API",
    
    // 工具和自动化
    "Puppeteer", "Selenium", "Task Runner", "Workflow Engine", "Cron Scheduler",
    "Build System", "CI/CD Pipeline", "Test Runner", "Deploy Agent", "Monitor Service",
    
    // AI和NLP相关
    "Embeddings Generator", "Vector Search", "Semantic Router", "RAG Engine", "Text Summarizer",
    "Entity Extractor", "Sentiment Analyzer", "Translation API", "Speech Recognition",
    "LLM Cache", "Token Counter", "Chat Memory", "Prompt Template", "Knowledge Graph",
    
    // 媒体处理
    "Image Generator", "Audio Processor", "Video Analyzer", "Media Converter",
    "Speech Synthesizer", "Content Moderator", "Style Transfer", "3D Renderer",
    
    // 开发工具
    "Code Repository", "Language Server", "Static Analyzer", "Debugger Interface",
    "Schema Validator", "API Gateway", "Auth Provider", "Identity Manager",
    
    // 特定领域
    "Finance Analyzer", "Health Processor", "Legal Assistant", "Science Data",
    "Academic Research", "Education Platform", "Game Engine", "Social Network"
  ];
  
  // 创建示例服务器
  const servers = [];
  const existingServers = [
    {
      id: "filesystem",
      name: "Filesystem",
      description: "安全的文件操作系统，具有可配置的访问控制",
      provider: "modelcontextprotocol",
      type: "official",
      tags: ["storage", "security"],
      documentation: "https://mcp.so/filesystem"
    },
    {
      id: "github",
      name: "GitHub",
      description: "仓库管理、文件操作和GitHub API集成",
      provider: "modelcontextprotocol",
      type: "official",
      tags: ["vcs", "github"],
      documentation: "https://mcp.so/github"
    },
    {
      id: "postgresql",
      name: "PostgreSQL",
      description: "只读数据库访问，支持架构检查",
      provider: "modelcontextprotocol",
      type: "official",
      tags: ["database", "storage"],
      documentation: "https://mcp.so/postgresql"
    },
    {
      id: "puppeteer",
      name: "Puppeteer",
      description: "浏览器自动化和网页抓取",
      provider: "modelcontextprotocol",
      type: "official",
      tags: ["automation", "scraping"],
      documentation: "https://mcp.so/puppeteer"
    },
    {
      id: "brave-search",
      name: "Brave Search",
      description: "使用Brave的搜索API进行网络搜索",
      provider: "modelcontextprotocol",
      type: "official", 
      tags: ["search", "api"],
      documentation: "https://mcp.so/brave-search"
    }
  ];
  
  // 添加现有服务器
  servers.push(...existingServers);
  
  // 添加更多生成的服务器
  for (let i = servers.length; i < count; i++) {
    let name;
    
    if (i < serverNames.length) {
      name = serverNames[i];
    } else {
      name = `MCP Server ${i + 1}`;
    }
    
    const id = name.toLowerCase().replace(/\s+/g, '-');
    
    // 生成随机标签 (1-4个)
    const randomTags = [];
    const tagCount = Math.floor(Math.random() * 3) + 1;
    for (let j = 0; j < tagCount; j++) {
      const tag = tags[Math.floor(Math.random() * tags.length)];
      if (!randomTags.includes(tag)) {
        randomTags.push(tag);
      }
    }
    
    // 决定类型（70%社区，30%官方）
    const type = Math.random() < 0.3 ? 'official' : 'community';
    
    // 为服务器生成不同的provider
    const provider = type === 'official' 
      ? 'modelcontextprotocol' 
      : `community-provider-${Math.floor(Math.random() * 10) + 1}`;
    
    // 随机生成支持的语言
    const languageCount = Math.floor(Math.random() * 3);
    const supportedLanguages = [];
    for (let j = 0; j < languageCount; j++) {
      const lang = languages[Math.floor(Math.random() * languages.length)];
      if (!supportedLanguages.includes(lang)) {
        supportedLanguages.push(lang);
      }
    }
    
    // 生成更详细的描述
    const descriptionParts = [
      `${name}是一个专门用于${randomTags.join('和')}的MCP服务器`,
      `提供高性能的${randomTags[0]}功能`,
      `支持${supportedLanguages.length > 0 ? supportedLanguages.join('和') : '多种编程语言'}`,
      type === 'official' ? '由官方团队维护' : '由社区贡献'
    ];
    const description = descriptionParts.join('，') + '。';
    
    // 创建服务器对象
    const server = {
      id,
      name,
      description,
      provider,
      type,
      tags: randomTags,
      url: `https://mcp.so/${id}`,
      github: Math.random() > 0.5 ? `https://github.com/mcp-community/${id}` : undefined,
      documentation: `https://docs.mcp.so/${id}`,
      language: supportedLanguages.length > 0 ? supportedLanguages : undefined
    };
    
    servers.push(server);
  }
  
  return { servers };
}

// 主函数
async function main() {
  try {
    // 获取服务器数据
    const data = await fetchServersFromMCP();
    
    // 保存到文件
    fs.writeFileSync(outputFilePath, JSON.stringify(data, null, 2), 'utf8');
    
    console.log(`成功保存了 ${data.servers.length} 个服务器的数据到 ${outputFilePath}`);
  } catch (error) {
    console.error('运行脚本时发生错误:', error);
  }
}

// 执行主函数
main(); 